Linear Regression from Scratch

Import libraries

import numpy as np
import random
import matplotlib.pyplot as plt

Data

X = [*range(1, 51)]
Y = list(map(lambda x: 2 * x + 5, X))

Univariate Regression

\(h() = X + b\)

MSE cost function

\((h(x) - y)^2\)

Gradient Descent

repeat {

    Ø = Ø - ∆J(Ø) = Ø - LR*1/m * sum((h(Ø, b) - Y)*X)    
    
    b = b - ∆J(b) =  b - LR*1/m * sum((h(Ø, b) - Y))
}

def mse(y_true, y_pred):
    cost = 0
    m = len(y_pred)
    for i in range(m):
        cost += (y_pred[i] - y_true[i]) ** 2
    return cost / (2 * m)


def der_mse(y_true, y_pred):
    der_cost = 0
    m = len(y_pred)
    for i in range(m):
        der_cost += y_pred[i] - y_true[i]
    return der_cost


def predict(x):
    return w * x + b

# Intialization of variables

m = len(X)
LR = 0.01
w, b = 0, 0.1

epochs = 10000
# Training

total_cost = []
for i in range(epochs):
    y_pred = []
    epoch_cost = []
    for num, data in enumerate(zip(X, Y)):
        x, y = data
        y_pred = []
        y_pred.append(w * x + b)

        cost = mse(Y[num : num + 1], y_pred)
        epoch_cost.append(cost)
        der_cost = der_mse(Y[num : num + 1], y_pred)

        w -= LR * (1 / m) * der_cost * x
        b -= LR * (1 / m) * der_cost

    total_cost.append(np.mean(epoch_cost))
    if i % 500 == 0:
        print(f"epoch:{i}\t\tcost:{cost}")

epoch:0     cost:0.024546020195931887
epoch:500       cost:0.0035238913511105277
epoch:1000      cost:0.0004771777468473895
epoch:1500      cost:6.461567040474519e-05
epoch:2000      cost:8.749747634800157e-06
epoch:2500      cost:1.1848222450189964e-06
epoch:3000      cost:1.604393419109384e-07
epoch:3500      cost:2.1725438173628743e-08
epoch:4000      cost:2.9418885555175706e-09
epoch:4500      cost:3.983674896607656e-10
epoch:5000      cost:5.3943803161575866e-11
epoch:5500      cost:7.30464704919418e-12
epoch:6000      cost:9.891380608202818e-13
epoch:6500      cost:1.3394131683086816e-13
epoch:7000      cost:1.8137281109430194e-14
epoch:7500      cost:2.4560089530711338e-15
epoch:8000      cost:3.3257381016463754e-16
epoch:8500      cost:4.5034718706313674e-17
epoch:9000      cost:6.09814092196085e-18
epoch:9500      cost:8.25761584212193e-19

predict(2), predict(9)

(8.999999990490096, 22.999999991911498)

w, b

(2.000000000203057, 4.999999990083981)

plt.plot(total_cost)
plt.show()